{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import os\n",
    "import sys\n",
    "sys.path.append('../')\n",
    "from utils import get_score\n",
    "import torch as t \n",
    "import numpy as np\n",
    "from glob import glob\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27\n"
     ]
    }
   ],
   "source": [
    "from glob import glob\n",
    "data_root=\"/data_ssd/zhihu/result/search_stack/\"\n",
    "files_path=glob(data_root+\"*val.pth\")\n",
    "files_path.sort()\n",
    "print len(files_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22\n"
     ]
    }
   ],
   "source": [
    "files_weight1=[]\n",
    "initial_weight=[]\n",
    "for file in files_path:\n",
    "    if 'weight5' not in file:\n",
    "        files_weight1.append(file)\n",
    "        if 'MultiModel' in file:\n",
    "            initial_weight.append(5)\n",
    "        else:\n",
    "            initial_weight.append(1)\n",
    "print len(files_weight1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/data_ssd/zhihu/result/search_stack/CNNText_tmp0.4024_char_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/CNNText_tmp0.4109_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/DeepText0.4103_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/DeepText_word_0.410051_aug_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/FastText3_word_weight1_0.40853_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/LSTMText0.4031_char_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/LSTMText0.4119_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/LSTMText0.41368_aug_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/MultiCNNTextBNDeep_aug_char_0.38738_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/MultiModel2_0.42560_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModel2w2c_0.4213_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModel_0.4171859_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModel_0.41987_word_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModel_0.42169_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModelall_0.4235_aug_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModelallfast_0.41652_aug_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/MultiModelallfast_0.419088_aug_val.pth 5\n",
      "/data_ssd/zhihu/result/search_stack/RCNN0.39854_aug_char_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/RCNN0.41344_aug_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/RCNN_0.4037_char_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/RCNN_0.4115_word_val.pth 1\n",
      "/data_ssd/zhihu/result/search_stack/inception0.41254_aug_word_val.pth 1\n"
     ]
    }
   ],
   "source": [
    "for f,w in zip(files_weight1,initial_weight):\n",
    "    print f,w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_num=10\n",
    "probs=[t.load(r).float() for r in files_weight1[:model_num]]\n",
    "test_data_path='/home/a/code/pytorch/zhihu/ddd/val.npz'\n",
    "index2qid = np.load(test_data_path)['index2qid'].item()\n",
    "label_path=\"/home/a/code/pytorch/zhihu/ddd/labels.json\"\n",
    "with open(label_path) as f: \n",
    "      labels_info = json.load(f)\n",
    "qid2label = labels_info['d']\n",
    "true_labels = [qid2label[index2qid[2999967-200000+ii]] for ii in range(200000)]\n",
    "del labels_info\n",
    "del qid2label\n",
    "del index2qid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def target(args):\n",
    "    r=0\n",
    "    for r_,k_ in enumerate(args):\n",
    "        if r_<model_num:\n",
    "            r +=k_*probs[r_]\n",
    "        else:\n",
    "            tmp=t.load(files_path[r_]).cuda().float()\n",
    "            r=r+k_*tmp.cpu()\n",
    "    result = r.topk(5,1)[1]\n",
    "    predict_label_and_marked_label_list = [[_1,_2] for _1,_2 in zip(result,true_labels)]\n",
    "    score,_,_,_ = get_score(predict_label_and_marked_label_list)\n",
    "    print (args,score,_)#list_space = [hp.uniform('a',0,1),hp.uniform('b',0,1)]\n",
    "    return -score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "((0.5767518711990203, 1.0535449863681772, 1.387628321556626, 0.8174540261504482, 0.46505220685010107, 0.84297799521111, 0.6365171155277725, 1.1619862524737483, 1.1629124711170251, 0.794240785899519, 1.3500216705996364, 0.7161582546569272, 0.8632489643123853, 1.0658517932708935, 0.7141997070591544, 1.1636288618440052, 0.8801467587212355, 0.9337439124613484, 0.7588341161043034, 1.0673229231836008, 0.9405260442990232, 1.0030074530136603, 1.233610926900131, 1.072457858205536, 0.8331957511909095, 0.8314988827650005, 1.3867110853019984), 0.4280465816350192, [118235, 66455, 43228, 29651, 21732])\n",
      "((0.9564174440498158, 1.0489559058574427, 0.7518749608094916, 1.0320464666633937, 0.9422874525955014, 1.0165090602024747, 1.0795518519765972, 0.8555932208868637, 1.2167276710281532, 1.419475183806352, 0.8838074782881737, 0.9468634334505386, 0.9691711235479938, 1.2110952652578792, 0.7660644586038812, 1.0678274437017292, 1.2004882120913785, 0.5873016262887354, 0.9972641287871216, 1.232087128048666, 0.5932669739562024, 0.8098430856930219, 1.2051749413279973, 0.6420124200617219, 1.0545220992911364, 1.090085158125075, 1.0434103800999925), 0.42835162928900966, [118317, 66486, 43171, 29730, 21802])\n",
      "((1.0396513912630647, 0.8152262036738671, 1.0151365556914107, 1.2222465705150265, 0.9237941653792502, 0.9145421492980752, 0.6709491327992462, 1.2240849176199395, 1.0882775208707203, 0.869310447972495, 0.8248709664979141, 1.0551567188255369, 0.9795312841684296, 1.1384864120379838, 0.6615855975033067, 0.9041668420703758, 0.9409216973018301, 1.1963661170777584, 1.0462748877740031, 1.0602561544349756, 1.219934932274373, 1.1146465247368262, 1.1756708688103288, 0.8091468678973395, 0.9344045282007267, 0.965189294016293, 0.7383995097742351), 0.4284971950755775, [118304, 66546, 43253, 29710, 21796])\n",
      "((1.0436207282403736, 1.1177847631655402, 0.7334644864020412, 0.5945583990768201, 1.1372775086544988, 0.9632482066449859, 1.185226546600236, 0.8547364731457139, 1.30567736183814, 1.0110489855369813, 1.156395824385718, 1.1807329353099296, 1.082701789584947, 0.9506669823106011, 0.8869190234648182, 1.3647780699602885, 0.9039268562212461, 0.9460467404143632, 0.49440395906757384, 1.2964158337990341, 0.9153578125760934, 0.7129527994158111, 1.1121794477398834, 0.7173885303546791, 0.8230543892941533, 1.060142871834187, 0.8274794389621235), 0.4284150098949782, [118296, 66537, 43173, 29779, 21768])\n",
      "((0.5603631370737425, 0.8877480056408759, 0.6558796529976378, 0.9886597888439004, 0.7477189610940553, 0.9046442608037218, 1.1992797282334795, 0.9588351253009877, 0.8294738023991932, 0.8437792101171209, 0.726724382887004, 1.2479942869290668, 0.683801073639847, 1.2098993886177676, 0.880619627297341, 0.7743649769768184, 1.096963809518645, 1.0204426915730789, 0.6681841600547878, 0.8049403488765382, 0.7980888274225503, 0.9599736009871855, 0.9897139328258182, 1.00187041793589, 0.9055640200005174, 1.4299287953224522, 0.9083399053181029), 0.4280640672955356, [118288, 66488, 43152, 29569, 21804])\n",
      "((0.8181619176741017, 0.9603357616332574, 0.8529903017325247, 0.966414166751806, 1.1477376869155627, 1.1243840294964045, 1.0927341716877685, 0.7462533057566294, 1.070288021044587, 0.9613682629977099, 1.3379171222667483, 0.9797534573713241, 1.0736052052577671, 1.2015391766065553, 1.1999794167439248, 1.203713309438228, 0.6152656113672589, 1.1720092968894404, 0.9968549106073766, 0.7980843446738006, 1.0951280389429952, 0.8181570938154149, 1.002099384294379, 1.182470659184026, 1.2146632196270613, 0.524237921153635, 1.024643251448422), 0.4281804320010674, [118283, 66440, 43183, 29726, 21759])\n",
      "((1.0960666582614125, 1.0579464482932797, 0.7834675994465389, 0.7903979165219251, 1.0413698326079688, 0.9456771022668258, 1.2255402916263032, 0.981313398006131, 0.852836243313798, 1.0756165440565637, 0.9403243119778307, 1.1503181341281592, 1.3105311830651543, 0.9348305757304137, 1.145807479022634, 1.1539449084733495, 0.9299528722380997, 0.6977885489944271, 1.1342172639449408, 1.389700852172688, 1.125297312045533, 0.9052773530037408, 0.9275938410331107, 1.2350001740419887, 0.8817933545128104, 0.8747652188780648, 1.2708864419865462), 0.4281017406924649, [118295, 66551, 43147, 29695, 21630])\n",
      "((1.0909774724122734, 1.1632946861879934, 0.8330937007459174, 1.2141352874659104, 1.3750239953251706, 0.8589485022199985, 1.0099166146388197, 0.9883074696777358, 1.0022839542828343, 0.776102193528407, 1.3413938899307376, 0.5692934122989379, 0.7176046834721495, 1.191582367245624, 0.7117175367466215, 0.9969798276049461, 0.672524361937618, 1.1569266954256807, 0.8912431960524776, 1.1350824964340147, 0.9726674348820902, 0.9947130191981621, 1.1323252165215676, 1.2500124809804412, 0.8858636818693135, 0.9011142886592992, 0.8718946284173767), 0.428951938455713, [118390, 66534, 43289, 29789, 21924])\n",
      "((0.9361006325341297, 0.7177671821986886, 1.1179556311823295, 0.861709780027206, 0.9429927555188609, 0.8821795069774191, 0.8410789772892293, 1.155421410686322, 0.8514129481784698, 1.192912860121629, 1.0636717336781352, 0.9522577242903932, 0.893136774708443, 1.2393469304819282, 1.3127825950348493, 0.9358325901097644, 1.2831212486148287, 1.1345971754491724, 0.8577949155879633, 0.9371835106885185, 0.6525743122175998, 0.9736165404423865, 1.4273092586434721, 0.9429513420478964, 0.7745209654177264, 1.0321206862261756, 1.2045688075773624), 0.42829769905968956, [118310, 66550, 43140, 29710, 21752])\n",
      "((1.0375872975886782, 1.1266258135132174, 1.257528811908275, 0.7126774977636814, 1.0350254091244158, 1.1450919600018103, 1.3314230648304846, 1.720986718975292, 0.5611567163165201, 1.1446875287185685, 1.046818745735875, 1.095238785121044, 1.0865878836682188, 1.090712317508523, 0.849033186686565, 0.8394915422381706, 1.1063798797082427, 1.0891893199379992, 1.0443234852412517, 1.3244008469618627, 0.9265194841669582, 0.8597205704217158, 0.7752412053380625, 0.9093594787023889, 0.7430527628256622, 1.11777911727874, 0.5899281435946844), 0.4287229616371521, [118322, 66623, 43187, 29770, 21866])\n",
      "((1.0742605955227833, 1.0682419526997646, 0.727248995316519, 0.7716229587350121, 1.1041876521277691, 0.8145663783524189, 0.9753185524861384, 1.2894101806832785, 1.1082271222350224, 1.277231681915051, 0.7337335528480802, 0.837210928876828, 1.0336346098160607, 1.1998204854833125, 1.270638179818906, 1.007861567287892, 0.9060395704204294, 1.0911707062348353, 1.008795433431396, 0.9558740498522629, 0.9587532819963107, 1.0942540727296848, 0.6038289863254356, 0.839704420058772, 1.1006514541954684, 1.14227088756331, 0.7270124183449314), 0.4286704252183209, [118320, 66563, 43291, 29682, 21876])\n",
      "((0.921355616535115, 1.2635944032433883, 1.163549280639701, 1.1677987169774517, 1.279552847964594, 0.9709836238348459, 1.1374443724855785, 1.2567197074516967, 0.7707326898951614, 0.8490424278697067, 0.576091026633405, 1.2915227921198216, 0.9194392652640613, 0.5120469491134019, 1.0505450689751525, 1.0803489437604752, 0.4217651963615686, 0.9931542979917196, 1.0521120733408396, 1.0967711532975704, 1.0311241977368202, 1.061097279261768, 0.916071883378249, 1.1078959807136524, 1.0761781897514162, 1.1916623122851995, 1.038368500785104), 0.4281014997593507, [118232, 66555, 43140, 29584, 21827])\n",
      "((0.8363859998153391, 0.968392492715505, 1.0631838754618983, 0.918861158275894, 1.017703394393451, 1.0764602173391697, 0.7210739477309421, 0.8485620548570673, 1.1509960691464665, 0.8722613732556246, 1.0230183780428093, 0.8109473776435979, 0.5205968403323541, 1.1966119229423686, 1.1782735765520647, 0.8215029549887057, 0.9053541058426728, 0.8117809688278214, 0.9177601828703651, 0.5734335667743669, 0.796373298455189, 0.969474300743803, 1.0755821421164158, 1.0439930428906756, 1.2383220222636562, 1.099832276043002, 1.2392361385601287), 0.4281050659660867, [118273, 66447, 43169, 29620, 21830])\n",
      "((1.0296686961406414, 1.2782244853907003, 1.2501647640675655, 1.2403838469104085, 1.0780295046650372, 1.0587047252750859, 1.2458020016586324, 1.2789441028099233, 1.0804801853394084, 1.1624201004390586, 1.2904461028210137, 1.003599590510022, 1.1945289180637169, 0.8397214333142977, 1.0264040833492132, 0.7386697396726365, 1.308397507493346, 1.0017690927125833, 0.9539847918439729, 0.6697360003518279, 0.8950169462219368, 1.126775843327582, 0.9759219393743628, 0.8738735310325617, 0.8057288711494516, 1.0054094595440621, 0.44890014536963785), 0.4288343253436665, [118328, 66567, 43276, 29777, 21904])\n",
      "((0.8521361193934514, 1.1037378158469435, 0.8728092728092407, 0.8673405782797443, 1.1160843025850589, 0.973274034857726, 0.9493745273690213, 0.8934372038391502, 1.1363568636226888, 1.0686341742329142, 0.9296050024455486, 0.9410160038361773, 0.7674216786545758, 0.8418684298678523, 0.9141477350235993, 1.0280465534195466, 1.1163021413227046, 1.4259901790437957, 1.3059288757704905, 1.0041475141166456, 1.0624299808147348, 1.1976957811419418, 1.0397239597130472, 1.158171759202412, 0.9450307976884158, 1.1545976308902526, 0.7346935621252547), 0.42862579733103073, [118360, 66545, 43298, 29707, 21778])\n",
      "((1.0049306680168337, 1.125106791722849, 1.2659471196498153, 1.1244502188839252, 1.0104630661693734, 1.053844199352109, 1.0344022756855036, 1.022972521164066, 1.1048288075861075, 0.833568825038338, 1.332337527265459, 1.3312288015392817, 1.1112902678961725, 0.9936867963378746, 1.3152705126000823, 0.7967982127653749, 0.9948443521348109, 1.262346599887762, 0.809348771595009, 0.7653213088403509, 1.050741951969147, 0.6667091702743495, 1.2695060878119722, 0.8687059429446946, 0.7652879831172674, 0.6051083028637301, 1.0235813580449669), 0.4281648492277763, [118237, 66495, 43141, 29742, 21773])\n",
      "((0.9439444089496458, 1.2643502738479635, 1.0981549682783176, 1.1535473276857933, 0.8590885431465769, 0.803822120317691, 1.2364706007873727, 0.999995817026468, 0.7616972718129256, 1.0647190703593277, 1.031256982080224, 0.7003975968936051, 1.4249272027399031, 0.9661740669923161, 1.0784221422100273, 0.7222909036907985, 0.9759302968975554, 0.9623363581963587, 0.8438962956706538, 0.9799820756758097, 1.0631232064584024, 1.1109672420454957, 0.8180911091130315, 0.3740752190642169, 0.6369347813943915, 1.1972483002232597, 1.0087215906713636), 0.42866310837214805, [118302, 66575, 43170, 29762, 21926])\n",
      "((1.1448566710143657, 1.1344063780038023, 1.0438750895405038, 1.0471380245152693, 1.3959556951404437, 0.8114342366840643, 0.8227616315983749, 1.228760836327871, 0.7601976432848806, 0.9140147177692148, 1.1382896942701592, 1.1552555964647135, 0.8146841628603113, 0.934080701839752, 0.8406695574084118, 0.6392142015240838, 1.1112428207739713, 1.1454791600790009, 0.7422563699227398, 1.306223598901489, 1.0056352843819367, 1.0430851566572166, 0.956723270825816, 1.14072237200691, 0.876572010416691, 1.0420998013009128, 1.200946583651936), 0.42822724488062136, [118258, 66547, 43165, 29643, 21811])\n",
      "((0.9682900034089849, 0.8944788565242395, 1.1424695733472774, 1.2097622579405274, 1.1838163654766434, 0.7712945895802327, 0.9987356706877797, 1.0459832860702756, 1.169801749407878, 1.3835554994368156, 0.9262485000863981, 1.0239760837079603, 1.1642004660326255, 0.7171660380110471, 1.2198435336331255, 0.9914586253312075, 1.0579685211673633, 1.165435321457424, 1.139935380380353, 1.1334999024891825, 0.8217591931057884, 0.8399501402618965, 0.9128560075182771, 0.831783823981961, 0.9368891285033222, 1.0695093106544433, 0.9907731908584041), 0.4283486770466729, [118284, 66513, 43187, 29695, 21831])\n",
      "((0.7142889794136915, 1.1857323261893113, 0.7583448946957471, 0.8586304369265904, 1.0173651383657554, 1.1475195360898676, 1.1430776079259468, 0.7865028816271515, 1.1976094401221742, 0.833036936596818, 1.1647559554604765, 0.9057551056279207, 0.8774045640984214, 1.316259711656861, 0.8826876196373359, 0.8754813876423517, 0.7550863115958579, 0.9839947093718612, 1.1694089035085011, 0.8313909263089377, 0.8040666063950112, 1.138072121288114, 0.9633824760139279, 1.1394564569892485, 0.9273606444547856, 1.1701525924936067, 0.9874737029270793), 0.4284862218442231, [118319, 66515, 43221, 29741, 21805])\n",
      "((1.133586351724914, 1.50553706140452, 1.5929445904941075, 1.6496567173923746, 1.5461892161064401, 0.579468566919412, 1.4991573358634755, 1.409549875309423, 1.0049633153704678, 0.5341050477510588, 1.4130295160266297, 0.3245984799285148, 1.2110830864417623, 1.5912282713299675, 0.25685591462716617, 0.4525979977623273, 1.450772134823792, 1.2576510831904293, 0.9241870903365496, 0.6190777133510634, 0.8729928252882753, 1.4687202014309415, 1.1446305290031749, 1.933303416480393, 0.7145319840105882, 0.9722557373354799, 0.09723813892992061), 0.4292862660939269, [118306, 66699, 43205, 29907, 22069])\n",
      "((1.2265553712293205, 1.6853060752500384, 1.7389560687469239, 1.8883074298584193, 1.8901796356028546, 0.3986334535773056, 1.7210304536348418, 1.4316303457558086, 0.9918747729369464, 0.07991582970906486, 1.915426191300876, 0.028778640687859847, 0.6338606696712282, 1.8613689024334066, 0.004502871431663447, 0.26396412744210535, 1.6652790614683415, 1.3217547537874714, 0.8911080211044674, 0.41949152360499065, 0.8588674070125663, 1.8227498540466323, 1.1368839156627115, 2.1203560639832935, 0.3512101663565764, 0.9266671560857133, -0.10212785221932139), 0.42909729518875406, [118317, 66598, 43100, 29958, 22083])\n",
      "((1.2957942777340852, 1.8956587348660159, 1.8797880806787033, 1.927135827524312, 2.4082382955373647, 0.18247199406812306, 1.9564552728098694, 1.4400604208311971, 0.9892161857392544, -0.24040183615693123, 2.3935525670702886, -0.12014519550697775, 0.501679629581739, 2.3401681585549987, -0.1538140232877496, -0.12678502715580586, 1.8750306814642714, 1.3470638798930443, 0.9252222081125101, -0.0798725030783699, 0.8584864356120623, 1.85936180701325, 1.1488598487395656, 2.4589601239251224, 0.02108568849425896, 0.9429424379970861, -0.28158514187764133), 0.4282534603788571, [118303, 66475, 42912, 29667, 22095])\n",
      "((1.2306384742422627, 1.5909741893344214, 1.6944812993290441, 1.662285856895472, 1.895188053580105, 0.4725550723728165, 1.645441796088941, 1.4472204835494829, 0.9931597514466172, 0.2755808147854392, 1.8760830386866907, 0.23087505325727006, 0.3218643496752165, 1.7900053810851841, 0.17994814506279125, 0.2760878875655698, 1.584523623473219, 1.6695897765751768, 0.8829682964258307, 0.3011883538184936, 0.7483510176453497, 1.5476062911215882, 1.2890598820416812, 1.8538176149211465, 0.2846176072545592, 0.9480514148776051, 0.06505422195345575), 0.42937608222769774, [118332, 66628, 43147, 29937, 22213])\n",
      "((1.2033203118537514, 1.5634683208836848, 1.6549648629083618, 1.65408620526703, 1.6596764430486408, 0.5116633970668553, 1.565428546007525, 1.447746336491962, 0.967363601061701, 0.4745222906112492, 1.729370608593019, 0.2603776176235985, -0.09563969616839424, 1.623186280320116, 0.21816912492833568, 0.3279947825081814, 1.6094399538232784, 2.019239373147481, 0.8595801929413844, 0.23634444661585755, 0.7233506864788657, 1.5415140657754784, 1.3012022620536692, 1.830739831579588, 0.2966987618531856, 0.9806570602058069, 0.07402629803182295), 0.42944389111657866, [118323, 66661, 43174, 29969, 22178])\n"
     ]
    }
   ],
   "source": [
    "max_evals=100\n",
    "from hyperopt import hp, fmin, rand, tpe, space_eval\n",
    "list_space = [hp.normal('a'+str(rr),1,0.2) for rr in range(len(files_path))]\n",
    "from hyperopt import Trials\n",
    "trials_to_keep=Trials()\n",
    "best = fmin(target,list_space,algo=tpe.suggest,max_evals=max_evals, trials = trials_to_keep)\n",
    "output = open('trials_to_keep__weight1_model'+'.pkl', 'wb')\n",
    "pickle.dump(trials_to_keep, output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
